{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import pathlib\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.support.ui import Select\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.common import exceptions\n",
    "import json\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pref_chrome = {\n",
    "            'profile.default_content_settings.popups': 0,\n",
    "            'profile.default_content_settings.automaticDownloads': 0,\n",
    "            'download.default_directory': 'I:\\\\中转\\\\'\n",
    "        }\n",
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument('disable-infobars')\n",
    "options.add_argument('lang=zh_CN.UTF-8')\n",
    "options.add_experimental_option('prefs', pref_chrome)\n",
    "options.add_extension(r\"D:\\Python\\IPython\\information\\DogAuthExtension.crx\")\n",
    "driver = webdriver.Chrome(chrome_options=options)\n",
    "locator = (By.ID, 'header')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "正在登录……\n",
      "登录成功！\n"
     ]
    }
   ],
   "source": [
    "\"\"\"登录成功到情报通网站\"\"\"\n",
    "driver.get(r'http://qbt.ecdataway.com/site/login')\n",
    "driver.implicitly_wait(10)\n",
    "driver.find_element_by_id('LoginForm_username').send_keys('森马上海服饰')\n",
    "driver.implicitly_wait(10)\n",
    "driver.find_element_by_id('LoginForm_password').send_keys('a123')\n",
    "driver.find_element_by_class_name('login_btn').click()\n",
    "try:\n",
    "    print('正在登录……')\n",
    "    WebDriverWait(driver, 60).until(EC.presence_of_element_located(locator))\n",
    "    driver.implicitly_wait(20)\n",
    "except exceptions.TimeoutException as e:\n",
    "    print(e.filename)\n",
    "    print('登录缓慢，请稍后……')\n",
    "print('登录成功！')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = r'http://qbt.ecdataway.com/shop'\n",
    "\"\"\"\n",
    ":param url: url\n",
    ":return: 根据指定的url选择所关注的20家品牌商的相关信息\n",
    "\"\"\"\n",
    "# locator_set1 = (By.ID, 'set1')\n",
    "js = '''document.getElementById('is_price').value=1;document.getElementById('is_csv').value=1;document.f1.submit();'''\n",
    "driver.get(url)\n",
    "try:\n",
    "    WebDriverWait(driver, 60).until(EC.presence_of_element_located(locator))\n",
    "    driver.implicitly_wait(10)\n",
    "#     time.sleep(10)\n",
    "except Exception as e:\n",
    "    print('1', e)\n",
    "# driver.find_element_by_name('a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_href = driver.find_elements_by_css_selector('a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib import parse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.find_elements_by_xpath(\"//*[@href]\")\n",
    "attribute = {}\n",
    "for link in all_href:\n",
    "    try:\n",
    "        href = link.get_attribute('href')\n",
    "        if 'index?sid=' in href:\n",
    "#             print(href)\n",
    "            name = parse.unquote(href).split('=')[2]\n",
    "            attribute[name] = href\n",
    "    except TypeError as e:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'茵曼旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=61886&nick=%E8%8C%B5%E6%9B%BC%E6%97%97%E8%88%B0%E5%BA%97',\n",
       " 'boylondon海外旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=190867697&nick=boylondon%E6%B5%B7%E5%A4%96%E6%97%97%E8%88%B0%E5%BA%97',\n",
       " 'npc官方旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=12094328&nick=npc%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97',\n",
       " 'burandoeno旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=63018547&nick=burandoeno%E6%97%97%E8%88%B0%E5%BA%97',\n",
       " 'clot官方旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=185987554&nick=clot%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97',\n",
       " 'viishow旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=1269800&nick=viishow%E6%97%97%E8%88%B0%E5%BA%97',\n",
       " '伊芙丽旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=7602672&nick=%E4%BC%8A%E8%8A%99%E4%B8%BD%E6%97%97%E8%88%B0%E5%BA%97',\n",
       " '拉夏贝尔官方旗舰': 'http://qbt.ecdataway.com/shop/index?sid=69344569&nick=%E6%8B%89%E5%A4%8F%E8%B4%9D%E5%B0%94%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0',\n",
       " 'veromoda官方旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=456560&nick=veromoda%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97',\n",
       " 'only官方旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=2238396&nick=only%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97',\n",
       " '毛菇小象': 'http://qbt.ecdataway.com/shop/index?sid=8170773&nick=%E6%AF%9B%E8%8F%87%E5%B0%8F%E8%B1%A1',\n",
       " 'ur官方旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=68414786&nick=ur%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97',\n",
       " '海澜之家官方旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=8759345&nick=%E6%B5%B7%E6%BE%9C%E4%B9%8B%E5%AE%B6%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97',\n",
       " 'gu官方旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=172809578&nick=gu%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97',\n",
       " '太平鸟男装旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=1269632&nick=%E5%A4%AA%E5%B9%B3%E9%B8%9F%E7%94%B7%E8%A3%85%E6%97%97%E8%88%B0%E5%BA%97',\n",
       " 'jackjones官方旗舰': 'http://qbt.ecdataway.com/shop/index?sid=1269592&nick=jackjones%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0',\n",
       " '森马官方旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=3433650&nick=%E6%A3%AE%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97',\n",
       " '优衣库官方旗舰店': 'http://qbt.ecdataway.com/shop/index?sid=61843&nick=%E4%BC%98%E8%A1%A3%E5%BA%93%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97'}"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attribute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "choose_attribute = ['viishow旗舰店', 'npc官方旗舰店', 'burandoeno旗舰店', 'boylondon海外旗舰店', 'clot官方旗舰店']\n",
    "item_name = choose_attribute[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "url_i = attribute[item_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(url_i)\n",
    "try:\n",
    "    WebDriverWait(driver, 60).until(EC.presence_of_element_located(locator))\n",
    "    driver.implicitly_wait(10)\n",
    "#     time.sleep(1)\n",
    "except exceptions.TimeoutException as e:\n",
    "    print('2', e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all_sidebar_category = driver.find_elements_by_id('sidebar_category')\n",
    "all_sidebar_category = driver.find_elements_by_css_selector('a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_of_clothe = {}\n",
    "for category in all_sidebar_category:\n",
    "    href = category.get_attribute('href')\n",
    "    if 'index?cid=' in href:\n",
    "        class_of_clothe[category.text] = href"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'男装': 'http://qbt.ecdataway.com/shop/index?cid=30',\n",
       " '服饰配件/皮带/帽子/围巾': 'http://qbt.ecdataway.com/shop/index?cid=50010404'}"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_of_clothe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "url_trend = list(class_of_clothe.values())[1] + '&type=2#fold_line'\n",
    "# list(class_of_clothe.values())\n",
    "driver.get(url_trend)\n",
    "try:\n",
    "    WebDriverWait(driver, 60).until(EC.presence_of_element_located(locator_set1))\n",
    "    driver.implicitly_wait(10)\n",
    "#     time.sleep(1)\n",
    "except exceptions.TimeoutException as e:\n",
    "    print('2', e)#f1 > table > tbody > tr:nth-child(5) > td:nth-child(2) > input:nth-child(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "locator_set1 = (By.ID, 'set1')\n",
    "period_start = Select(self.driver.find_element_by_id('period_start'))\n",
    "period_start.select_by_value('201707')\n",
    "period_last = Select(self.driver.find_element_by_id('period_last'))\n",
    "period_last.select_by_value('201712')\n",
    "# self.driver.find_element_by_name('is_mall').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#show_line_date').click()\n",
    "driver.find_element_by_css_selector('#f1 > table > tbody > tr:nth-child(6) > td:nth-child(2) > input:nth-child(2)').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_id('submit-button').click()\n",
    "try:\n",
    "    WebDriverWait(driver, 60).until(EC.presence_of_element_located(locator_set1))\n",
    "    driver.implicitly_wait(10)\n",
    "#     time.sleep(1)\n",
    "except exceptions.TimeoutException as e:\n",
    "    print('2', e)#f1 > table > tbody > tr:nth-child(5) > td:nth-child(2) > input:nth-child(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "table_text = driver.find_element_by_id('set1').text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['类别名称',\n",
       " '2017-07',\n",
       " '2017-08',\n",
       " '2017-09',\n",
       " '2017-10',\n",
       " '2017-11',\n",
       " '2017-12',\n",
       " '总计',\n",
       " '销售额',\n",
       " '销售额',\n",
       " '环比',\n",
       " '销售额',\n",
       " '环比',\n",
       " '销售额',\n",
       " '环比',\n",
       " '销售额',\n",
       " '环比',\n",
       " '销售额',\n",
       " '环比',\n",
       " '销售额',\n",
       " '帽子 78,680 67,676 -14.0% 25,760 -61.9% 560 -97.8% 10,360 1750.0% 43,057 315.6% 226,093',\n",
       " '总计 78,680 67,676 -14.0% 25,760 -61.9% 560 -97.8% 10,360 1750.0% 43,057 315.6% 226,093']"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_text.split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = []\n",
    "for y in [i.split(' ') for i in table_text.split('\\n')]:\n",
    "    for x in y:\n",
    "        s.append(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = s[20:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame([s[i: i+13] for i in range(0, len(s), 13)])\n",
    "df['店铺'] = item_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.to_excel('i:{}.xls'.format(item_name), index=False, encoding='utf-8', header=False)\n",
    "js = '''document.getElementById('is_price').value=1;document.getElementById('is_csv').value=1;document.f1.submit();'''\n",
    "driver.execute_script(js)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class_of_clothe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'class_of_clothe' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-85-d7a1dc0fb46a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mclass_of_clothe\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'class_of_clothe' is not defined"
     ]
    }
   ],
   "source": [
    "# for i in class_of_clothe.values():\n",
    "#     print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.get(r'http://qbt.ecdataway.com/shop/index?cid=16')\n",
    "# try:\n",
    "#     WebDriverWait(driver, 60).until(EC.presence_of_element_located(locator))\n",
    "#     driver.implicitly_wait(10)\n",
    "# #     time.sleep(1)\n",
    "# except exceptions.TimeoutException as e:\n",
    "#     print('2', e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "extend() takes exactly one argument (0 given)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-47-e77f9e40dc5b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0ms\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m: extend() takes exactly one argument (0 given)"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "41%2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
